# -*- coding: utf-8 -*-
import os
import scrapy
from scrapy_redis.spiders import RedisSpider

class IconSpider(RedisSpider):
    name = 'icon'
    # allowed_domains = ['http://sc.chinaz.com/tubiao/']
    # start_urls = ['http://sc.chinaz.com/tubiao/']
    redis_key = 'icon'
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop("domains", "")
        self.alllowed_domains = filter(None, domain.split(','))
        super(IconSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        '''
        获取图标分类
        :param response:
        :return:
        '''
        classify_name = response.xpath('//*[@class="pnglist1"]/a/span/text()').extract()
        classify_url = response.xpath('//*[@class="pnglist1"]/a/@href').extract()
        classify_list = []
        for i in range(len(classify_name)):
            classify_dict = {
                'classify_name': classify_name[i],
                'classify_url': 'http://sc.chinaz.com/tubiao/'+classify_url[i]
            }
            classify_list.append(classify_dict)

        for j in classify_list:
            yield scrapy.Request(
                url=j['classify_url'],
                callback=self.request_classify,
                meta={'huiger': j},
                dont_filter=True
            )

    def request_classify(self, response):
        '''
        请求某个分类，获取到响应，从中获取到当前页面的图标地址
        注意：当前页面有分页
        :param response:
        :return:
        '''
        data = response.meta['huiger']
        # 获取当前页面的图标链接
        link_name = response.xpath('//*[@class="pngblock imgload"]/li/p/a/@alt').extract()
        link_url = response.xpath('//*[@class="pngblock imgload"]/li/p/a/@href').extract()
        for i in range(len(link_name)):
            data['link_name'] = link_name[i]
            data['link_url'] = link_url[i]
            yield scrapy.Request(
                url=link_url[i],
                callback=self.request_link,
                meta={'huiger': data},
                dont_filter=True
            )

    def request_link(self, response):
        '''
        请求链接，获取当前链接下的图标url
        :param response:
        :return:
        '''
        data = response.meta['huiger']
        icon_urls = response.xpath('//*[@class="png_pic"]/img/@src').extract()
        for i in range(len(icon_urls)):
            data['icon_url'] = icon_urls[i]
            yield scrapy.Request(
                url=icon_urls[i],
                callback=self.save_icon,
                meta={'huiger': data},
                dont_filter=True
            )

    def save_icon(self, response):
        '''
        保存图标
        :param response:
        :return:
        '''
        data = response.meta['huiger']
        # 大分类名称
        fa_classify_name = data['classify_name']
        # 小分类名称
        son_classify_name = data['link_name']
        # 图片名称
        icon_name = data['icon_url'].split('/')[-1]
        # 首先判断大分类文件夹是否存在
        if os.path.exists(r'D:/code/站长素材图片下载/图标/' + fa_classify_name):
            # 其次判断小分类文件夹是否存在
            if os.path.exists(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name):
                with open(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name + '/' + icon_name,
                          'wb') as f:
                    f.write(response.body)
                    print('{}/{}/{}下载完成。。。'.format(fa_classify_name, son_classify_name, icon_name))

            else:
                os.mkdir(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name)
                with open(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name + '/' + icon_name,
                          'wb') as f:
                    f.write(response.body)
                    print('{}/{}/{}下载完成。。。'.format(fa_classify_name, son_classify_name, icon_name))
        else:
            os.mkdir(r'D:/code/站长素材图片下载/图标/' + fa_classify_name)
            if os.path.exists(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name):
                with open(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name + '/' + icon_name,
                          'wb') as f:
                    f.write(response.body)
                    print('{}/{}/{}下载完成。。。'.format(fa_classify_name, son_classify_name, icon_name))
            else:
                os.mkdir(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name)
                with open(r'D:/code/站长素材图片下载/图标/' + fa_classify_name + '/' + son_classify_name + '/' + icon_name,
                          'wb') as f:
                    f.write(response.body)
                    print('{}/{}/{}下载完成。。。'.format(fa_classify_name, son_classify_name, icon_name))
